/*
 * @(#)LuceneIndexUpgradeHandler.java
 *
 * Copyright (c) 2004 DCIVision Ltd
 * All rights reserved.
 *
 * This software is the confidential and proprietary information of DCIVision
 * Ltd ("Confidential Information").  You shall not disclose such Confidential
 * Information and shall use it only in accordance with the terms of the license
 * agreement you entered into with DCIVision Ltd.
 */
package com.dcivision.upgrade.handler;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Hits;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.FSDirectory;

import com.dcivision.dms.bean.DmsDocument;
import com.dcivision.dms.bean.DmsDocumentDetail;
import com.dcivision.dms.bean.DmsLocMaster;
import com.dcivision.dms.bean.DmsRoot;
import com.dcivision.dms.dao.DmsDocumentDAObject;
import com.dcivision.dms.dao.DmsDocumentDetailDAObject;
import com.dcivision.dms.dao.DmsLocMasterDAObject;
import com.dcivision.dms.dao.DmsRootDAObject;
import com.dcivision.framework.ApplicationException;
import com.dcivision.framework.GlobalConstant;
import com.dcivision.framework.SessionContainer;
import com.dcivision.framework.SystemParameterConstant;
import com.dcivision.framework.SystemParameterFactory;
import com.dcivision.framework.Utility;
import com.dcivision.lucene.FieldNames;
import com.dcivision.lucene.FieldTypes;
import com.dcivision.lucene.IndexConverter;
import com.dcivision.lucene.IndexManager;
import com.dcivision.upgrade.UpgradeContext;

/**
 * Perform lucene index upgrade part of DMS module upgrade.
 * 
 * @author      Brook.Huang
 * @company     DCIVision Limited
 * @creation    11/07/2007
 * @version     $Revision: 1.1.2.3 $
 */
public class LuceneIndexUpgradeHandler extends UpgradeHandlerBase {

  private File oldLuceneIndexDir = null;

  private File indexDir = null;

  private String oldLuceneIndexPath = null;

  private boolean isUpdateFromOldLuceneIndex = false;

  private SessionContainer sessionContainer = null;

  private DmsDocumentDAObject dmsDocumentDAO = null;

  private DmsDocumentDetailDAObject dmsDocumentDetailDAO = null;

  private DmsLocMasterDAObject dmsLocMasterDAO = null;

  private DmsRootDAObject dmsRootDAO = null;

  private IndexSearcher indexSearcher = null;

  private int recordCount;

  public LuceneIndexUpgradeHandler(UpgradeContext upgradeContext) throws Exception {
    super(upgradeContext);

    isUpdateFromOldLuceneIndex = "true".equals(upgradeContext.getUpgradeRequest().getUpgradeParameter("isUpdateFromOldLuceneIndex")) ? true : false;
    
    if (isUpdateFromOldLuceneIndex) {
      oldLuceneIndexPath = (String) upgradeContext.getUpgradeRequest().getUpgradeParameter("oldLuceneIndexPath");
      
      // Verifies the valid file path, exit the upgrade if there is invalid file path found.
      oldLuceneIndexDir = new File(oldLuceneIndexPath);

      if (!oldLuceneIndexDir.exists() || oldLuceneIndexDir.isFile()) {
        oldLuceneIndexDir = null;
        throw new IllegalArgumentException("ERROR: Old lucene index folder is invalid.");
      }
    }
    
    indexDir = new File(SystemParameterFactory.getSystemParameter(SystemParameterConstant.INDEX_BASE_PATH));
    if (!indexDir.exists()) {
      indexDir = null;
      throw new IllegalArgumentException("ERROR: index base path is not exist.");
    }

    sessionContainer = (SessionContainer) upgradeContext.getUpgradeRequest().getUpgradeParameter(GlobalConstant.SESSION_CONTAINER_KEY);
    if (sessionContainer == null) {
      throw new IllegalArgumentException("ERROR: sessionContainer for DOC module upgrade cannt be null.");
    }
  }

  /**
   * @see com.dcivision.upgrade.handler.UpgradeHandlerBase#upgradeBegin()
   */
  protected void upgradeBegin() throws Exception {
  }

  /**
   * @see com.dcivision.upgrade.handler.UpgradeHandlerBase#upgradeEnd()
   */
  protected void upgradeEnd() throws Exception {

    // (1) Lucene Index Upgrade.
    // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    luceneIndexUpgrade();
  }

  private int[] luceneIndexUpgrade() throws Exception {

    upgradeContext.logReport("<p><font color=blue>START UPGRADE LUCENE INDEX</font><br>\n", null);

    int[] count = null;

    emptyFolder();

    recordCount = getRecordCountFromDB();

    count = reIndexLucene();

    upgradeContext.logReport("<font color=blue>UPGRADE LUCENE INDEX SUCCESS (TOTAL SUCCESSFUL COUNT: " + count[0] + ", <font color=red>FAILED COUNT: " + count[1] + "</font>)</font><br></p>\n", null);

    return (count);
  }

  ///////////////////////////////////////////////////////////////////////////
  ///
  /// Methods move from {@link com.dcivision.dms.core.LuceneUpdater}.
  ///
  ///////////////////////////////////////////////////////////////////////////

  // Do index lucene.
  private int[] reIndexLucene() throws Exception {

    int[] count = {0, 0};

    String[] fieldNames = new String[] { "D.ID", "D.DOCUMENT_NAME", "D.DOCUMENT_TYPE", "D.EFFECTIVE_START_DATE", "D.EFFECTIVE_END_DATE", "D.ROOT_ID", "D.REFERENCE_NO", "D.DESCRIPTION",
        "D.RECORD_STATUS", "D.ITEM_STATUS", "D.UPDATER_ID", "D.UPDATE_DATE", "D.USER_DEF_1", "D.USER_DEF_2", "D.USER_DEF_3",
        "D.USER_DEF_4", "D.USER_DEF_5", "D.USER_DEF_6", "D.USER_DEF_7", "D.USER_DEF_8", "D.USER_DEF_9", "D.USER_DEF_10", "V.ID",
        "V.ITEM_SIZE", "C.ID", "C.CONVERTED_NAME", "C.SEGMENT_NO", "C.EXT" };
    String[] fieldTypes = new String[] { "Integer", "String", "String", "Timestamp", "Timestamp", "Integer", "String", "String", "String", "String", "Integer",
        "Timestamp", "String", "String", "String", "String", "String", "String", "String", "String", "String", "String", "Integer",
        "Integer", "Integer", "String", "Integer", "String" };
    String orderBy = "D.ID ASC, V.ID DESC";

    // reindex document (lucene)
    Map basePathCache = new HashMap();
    IndexManager indexManager = null;
    try {
      indexManager = new IndexManager(sessionContainer, upgradeContext.getUpgradeConnection());

      int pageIndex = 0;
      int pageSize = 100;
      while (true) {
        if (pageIndex * pageSize > recordCount) {
          break;
        }

        List lstDocumentIds = new ArrayList();
        for (int i = 0; i < pageSize; i++) {
          lstDocumentIds.add(new Integer((pageIndex * pageSize) + i));
        }

        Map mapDocuments = getDmsDocumentDAO().getMapIdToDVCFieldValueArrayList(lstDocumentIds, fieldNames, fieldTypes, orderBy);

        for (int i = 0; i < lstDocumentIds.size(); i++) {
          Integer documentID = (Integer) lstDocumentIds.get(i);
          List lstDocuments = (List) mapDocuments.get(documentID);
          if (lstDocuments != null && !lstDocuments.isEmpty()) {
            Object[] fieldValues = (Object[]) lstDocuments.get(0);

            try {
              Integer rootID = (Integer) fieldValues[5];

              List documentDetailList = getDmsDocumentDetailDAO().getListByDocumentID(documentID);
              String content = getFullTextFromLucene(documentID);
              String basePath = (String) basePathCache.get(rootID);
              if (basePath == null) {
                basePath = getBasePath(rootID);
                basePathCache.put(rootID, basePath);
              }

              Document document = getDocumentByFullParam(fieldValues, content, documentDetailList, basePath);
              indexManager.addDocument(document);
              count[0]++;
              log.info("Success to add lucene : Document " + (String) fieldValues[1] + "(" + fieldValues[0] + ").");
              upgradeContext.logReport(null, "INDEX -- ID : " + fieldValues[0] + "; DOC NAME : " + fieldValues[1] + "<br>");
            } catch (Exception e) {
              count[1]++;
              log.warn("Document " + (String) fieldValues[1] + "(" + fieldValues[0] + ") fail to add lucene.", e);              
              upgradeContext.logReport(null, "<font color=\"red\">INDEX -- ID : " + fieldValues[0] + "; DOC NAME : " + fieldValues[1] + "</font><br>");
            }
          }
        }
        
        pageIndex++;
      }

      return count;
    } finally {
      indexManager.close(true);
    }
  }

  private void emptyFolder() {
    File[] files = indexDir.listFiles();
    for (int i = 0; i < files.length; i++) {
      File file = files[i];
      file.delete();
    }
  }

  // Get new document object by giving data.
  private Document getDocumentByFullParam(Object[] fieldValues, String content, List dmsDocumentDetailList, String basePath)
      throws Exception {
    Document document = new Document();
    // set index main information
    document.add(getField(FieldNames.UID, ((Integer) fieldValues[0]).toString()));
    document.add(getField(FieldNames.DOCUMENT_NAME, (String) fieldValues[1]));
    document.add(getField(FieldNames.DOCUMENT_NAME_SORT, (String) fieldValues[1]));
    document.add(getField(FieldNames.EFFECTIVE_START_DATE, Utility.isEmpty(fieldValues[3])?"":((Timestamp) fieldValues[3]).toString()));
    document.add(getField(FieldNames.EFFECTIVE_END_DATE, Utility.isEmpty(fieldValues[4])?"":((Timestamp) fieldValues[4]).toString()));
    document.add(getField(FieldNames.ROOT_ID, ((Integer) fieldValues[5]).toString()));
    document.add(getField(FieldNames.RECORD_STATUS, (String) fieldValues[8]));

    if (!Utility.isEmpty((String) fieldValues[6])) {
      document.add(getField(FieldNames.REFERENCE_NO, (String) fieldValues[6]));
    }
    if (!Utility.isEmpty((String) fieldValues[7])) {
      document.add(getField(FieldNames.DESCRIPTION, (String) fieldValues[7]));
    }
    if (!Utility.isEmpty((String) fieldValues[27])) {
      document.add(getField(FieldNames.EXTENSION, (String) fieldValues[27]));
    }
    if (!Utility.isEmpty(fieldValues[23])) {
      document.add(getField(FieldNames.ITEM_SIZE, ((Integer) fieldValues[23]).toString()));
    }
    if (!Utility.isEmpty((String) fieldValues[9])) {
      document.add(getField(FieldNames.ITEM_STATUS, (String) fieldValues[9]));
    }
    if (!Utility.isEmpty(fieldValues[10])) {
      document.add(getField(FieldNames.UPDATER_ID, ((Integer) fieldValues[10]).toString()));
    }
    if (!Utility.isEmpty(fieldValues[11])) {
      document.add(getField(FieldNames.UPDATE_DATE, ((Timestamp) fieldValues[11]).toString()));
    }

    // set default profile
    for (int i = 0; i < 10; i++) {
      if (!Utility.isEmpty((String) fieldValues[12 + i])) {
        document.add(getField(FieldNames.DEFAULT_INDEXES + i, FieldNames.DEFAULT_INDEXES, (String) fieldValues[12 + i]));
      }
    }

    // set physicalfile information
    String extension = (String) fieldValues[27];
    if (content != null) {
      document.add(getField(FieldNames.CONTENT, content));
    } else if (!Utility.isEmpty(extension) && IndexConverter.isAllowConvert(extension) && isDocumentType((String) fieldValues[2])) {
      try {
        Integer segmentNo = (Integer) fieldValues[26];
        File file = new File(basePath + "/" + (segmentNo != null ? "Document/segment" + segmentNo.toString() + "/" : "")
            + (String) fieldValues[25]);
        if (file.exists() && file.isFile()) {
          InputStream ins = null;
          try {
            ins = new FileInputStream(file);
            content = IndexConverter.getText(ins, extension);
            if (content != null) {
              document.add(getField(FieldNames.CONTENT, content));
            }
          } catch (Exception ex) {
            throw ex;
          } finally {
            try {
              ins.close();
            } catch (Exception ignore) {
            } finally {
              ins = null;
            }
          }
        } else {
          throw new ApplicationException("Document " + (String) fieldValues[1] + "(" + fieldValues[0] + ") physical file is missing.");
        }
      } catch (Exception ex) {
        log.warn("Document " + (String) fieldValues[1] + "(" + fieldValues[0] + ") convet to text failure by unknown errror.", ex);
        throw new ApplicationException(ex);
      }
    }

    // set user defined profile
    if (dmsDocumentDetailList != null && !dmsDocumentDetailList.isEmpty()) {
      String udfIndexDetailIds = "";
      for (int i = 0; i < dmsDocumentDetailList.size(); i++) {
        DmsDocumentDetail detail = (DmsDocumentDetail) dmsDocumentDetailList.get(i);
        if (i == 0) {
          document.add(getField(FieldNames.USER_DEFINED_INDEX_ID, detail.getUserDefinedFieldID().toString()));
        }
        udfIndexDetailIds += ((i == 0 ? "" : " ") + detail.getUserDefinedFieldDetailID());
        String value = (detail.getFieldValue() != null ? detail.getFieldValue() : (detail.getDateValue() != null ? detail.getDateValue()
            .toString() : (detail.getNumericValue() != null ? detail.getNumericValue().toString() : null)));
        if (!Utility.isEmpty(value)) {
          document.add(getField(FieldNames.USER_DEFINED_INDEXES + i, FieldNames.USER_DEFINED_INDEXES, value));
        }
      }
      document.add(getField(FieldNames.USER_DEFINED_INDEX_DETAIL_IDS, udfIndexDetailIds));
    }
    return document;
  }

  // Get lucene full text from old lucene index.
  private String getFullTextFromLucene(Integer documentID) throws IOException {
    if (isUpdateFromOldLuceneIndex) {
      Query termQuery = new TermQuery(new Term(FieldNames.UID, documentID.toString()));
      Hits hits = getIndexSearcher().search(termQuery);
      if (hits.length() > 0) {
        return hits.doc(0).get(FieldNames.CONTENT);
      }
    }
    
    return null;
  }

  // Get max update records count.
  private int getRecordCountFromDB() throws Exception {
    PreparedStatement psmt = null;
    ResultSet rs = null;
    try {
      int max = 0;
      String sql = "SELECT max(ID) FROM dms_document";
      psmt = upgradeContext.getUpgradeConnection().prepareStatement(sql, ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
      rs = psmt.executeQuery();
      if (rs.next()) {
        max = rs.getInt(1);
      }
      return max;
    } catch (Exception ex) {
      throw ex;
    } finally {
      try {
        rs.close();
      } catch (Exception ignore) {
      } finally {
        rs = null;
      }
      try {
        psmt.close();
      } catch (Exception ignore) {
      } finally {
        psmt = null;
      }
    }
  }

  private IndexSearcher getIndexSearcher() throws IOException {
    if (indexSearcher == null) {
      indexSearcher = new IndexSearcher(FSDirectory.getDirectory(oldLuceneIndexPath, false));
    }
    return indexSearcher;
  }

  private Field getField(String fieldName, String fieldValue) throws Exception {
    return new Field(fieldName, fieldValue, FieldTypes.getStore(fieldName), FieldTypes.getIndex(fieldName), FieldTypes
        .getTermVector(fieldName));
  }

  private Field getField(String fieldName, String typeName, String fieldValue) throws Exception {
    return new Field(fieldName, fieldValue, FieldTypes.getStore(typeName), FieldTypes.getIndex(typeName), FieldTypes
        .getTermVector(typeName));
  }

  private boolean isDocumentType(String documentType) throws Exception {
    return (DmsDocument.DOCUMENT_TYPE.equals(documentType) || DmsDocument.COMPOUND_DOC_TYPE.equals(documentType)
        || DmsDocument.EMAIL_DOC_TYPE.equals(documentType) || DmsDocument.ORIGINSLITY_EMAIL_TYPE.equals(documentType));
  }

  private String getBasePath(Integer rootID) throws Exception {
    return ((DmsLocMaster) getDmsLocMasterDAO().getObjectByID(
        ((DmsRoot) getDmsRootDAO().getObjectByIDIgnoreRecordStatus(rootID)).getLocID())).getLocPath();
  }

  private DmsDocumentDAObject getDmsDocumentDAO() {
    if (dmsDocumentDAO == null) {
      dmsDocumentDAO = new DmsDocumentDAObject(sessionContainer, upgradeContext.getUpgradeConnection());
    }
    return dmsDocumentDAO;
  }

  private DmsDocumentDetailDAObject getDmsDocumentDetailDAO() {
    if (dmsDocumentDetailDAO == null) {
      dmsDocumentDetailDAO = new DmsDocumentDetailDAObject(sessionContainer, upgradeContext.getUpgradeConnection());
    }
    return dmsDocumentDetailDAO;
  }

  private DmsLocMasterDAObject getDmsLocMasterDAO() {
    if (dmsLocMasterDAO == null) {
      dmsLocMasterDAO = new DmsLocMasterDAObject(sessionContainer, upgradeContext.getUpgradeConnection());
    }
    return dmsLocMasterDAO;
  }

  private DmsRootDAObject getDmsRootDAO() {
    if (dmsRootDAO == null) {
      dmsRootDAO = new DmsRootDAObject(sessionContainer, upgradeContext.getUpgradeConnection());
    }
    return dmsRootDAO;
  }
}
